package com.toutiao.nssue

import scala.math.sqrt
import java.util.Date
import java.net.URL
import scala.io.Source

import com.toutiao.score.Scorer

class Document(
		val name:String,
		val vs:Map[String,Double],
		val title:Array[String],
		val date:Date
    ) {
}

class RawDocument(
		val name:String,
		val title:String,
		val content:String,
		val vs:Map[String,Double],
		//val cut_title:Array[String],
		val uri:String,
		val date:Date
    ) {
}

// 做两件事
// 1. 关键词做Normalize
// 2. 对不需要处理的Doc提前过滤掉

class DocProcessor(val source:Map[String,Int],
				   val scorer: Scorer) {
  
  val X = 8.0
  val maxV = 55.0
  val minV = 5.0
  val maxN = 15
  
  def noNeed(u:String) = {
    val v = 
      try {
        new URL(u).getHost
      }
      catch {
        case _: Throwable => u
      }
    
    if (source.getOrElse(v,0) == 1 )
      true
    else
      false
  }
  
  // 对原始关键词的处理
  def norm(vs:Map[String,Double]) = {
    val us = vs.filter(_._2 > minV).toArray.
    			map(vv => {
    			  if (vv._2 > maxV)
    			    (vv._1,maxV)
    			  else
    			    vv
    			}).
    			sortBy(-_._2)
    val cnt = if (us.size > maxN) maxN else us.size
    val vvs = us.slice(0, cnt).toMap
    val s = if (vvs.size < maxN){
     vvs.values.map(v => v*v).sum+(maxN-vvs.size)*X*X
    }
    else {
      vvs.values.map(v => v*v).sum
    }
    val ss = sqrt(s)
    vvs.map(vv => {
      (vv._1,vv._2/ss)
    })
  }
  
  def process(rs:Array[RawDocument],m:Boolean = false) = {
    rs.filter(r => {
      val u = r.uri
      !noNeed(u)
    }).filter(_.vs.size > 5)
      .filter(_.title.trim.size > 0)
      .filter(_.content.trim.size > 0)
      .map(r => {
      val vs = 
        if (m) {
         norm(r.vs) 
        }
        else {
          scorer.score(r.title, r.content)
        }
      //val us = norm(vs)
      new Document(r.name,vs,r.title.split(" "),r.date)
    })
  }
  
}


object DocData {
  
  def getSource(location:String) = {
    val source = Source.fromFile(location,"UTF-8")
    val lineIterator = source.getLines
    val m = lineIterator.map(line => {
      val Array(s,i) = line.trim.split("\t")
    		  (s,i.toInt)
    	}
    ).toMap
    m
  }
  
}


//object testTest extends App {
//  
//  val u = "http://www.56.com/u51/v_MTM3NTMwOTI4.html"
//    
//    val v = 
//      try {
//        new URL(u).getHost
//      }
//      catch {
//        case _: Throwable => u
//      }
//      
//   println(v)
//  
//  
//}